-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
integrate chains sheet into crawler, handle exceptional cases and upd…
…ate test cases (#37) * Add COUNT_API_KEY to environment variables, refactor Discord and GitHub functions, and update test cases * Refactor YouTube video date retrieval to use YouTube Data API and improve schema validation * Add COUNT_API_KEY to .env.example for environment configuration * Refactor environment configuration to remove COUNT_API_KEY and update YouTube channel info function to use imported COUNT_API_KEY * Refactor YouTube functions to use YouTube Data API, improve schema validation, and memoize channel info retrieval * Refactor main function to restore YouTube metrics retrieval and clean up commented code * Refactor imports to use local ofetch module instead of external ofetch package * Update getTelegramGroupChatInfo to return memberCount and onlineCount * Remove ignoreResponseError option from getGithubRepositoryRelease function * Add null check for response in getGithubRepositoryRelease function
Showing
61 changed files
with
647 additions
and
388 deletions.
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,20 @@ | ||
| import { getGithubOrganizationInfo } from "@/functions/github/getGithubOrganizationInfo" | ||
| import { describe, expect, it, vi } from "vitest" | ||
|
|
||
| describe("getGithubOrganizationInfo function format tests", () => { | ||
| it("should match the expected response format", async () => { | ||
| describe("getGithubOrganizationInfo", () => { | ||
| it("base", async () => { | ||
| const organizationName = "base-org" | ||
|
|
||
| const response = await getGithubOrganizationInfo(organizationName) | ||
|
|
||
| expect(response.login).toBe(organizationName) | ||
| }) | ||
|
|
||
| it("bitcoin", async () => { | ||
| const organizationName = "bitcoin" | ||
|
|
||
| const response = await getGithubOrganizationInfo(organizationName) | ||
|
|
||
| expect(response.login).toBe(organizationName) | ||
| }) | ||
| }) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| import { getLastVideoDateFromYoutube } from "@/functions/youtube/getLastVideoDateFromYoutube" | ||
| import { describe, expect, it } from "vitest" | ||
|
|
||
| describe("getLastVideoDateFromYoutube", () => { | ||
| it("ethereum", async () => { | ||
| const channelId = "UC6rYoXJ_3BbPyWx_GQDDRRQ" | ||
|
|
||
| const response = await getLastVideoDateFromYoutube(channelId) | ||
|
|
||
| expect(response).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/) | ||
| }) | ||
|
|
||
| it.skip("algorand", async () => { | ||
| const channelId = "UCsda5E-IaXyUi8dzauyXypA" | ||
|
|
||
| const response = await getLastVideoDateFromYoutube(channelId) | ||
|
|
||
| expect(response).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/) | ||
| }) | ||
|
|
||
| it.skip("bnb", async () => { | ||
| const channelId = "UCG9fZu6D4I83DStktBV0Ryw" | ||
|
|
||
| const response = await getLastVideoDateFromYoutube(channelId) | ||
|
|
||
| expect(response).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/) | ||
| }) | ||
|
|
||
| it.skip("cardano", async () => { | ||
| const channelId = "UCbQ9vGfezru1YRI1zDCtTGg" | ||
|
|
||
| const response = await getLastVideoDateFromYoutube(channelId) | ||
|
|
||
| expect(response).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/) | ||
| }) | ||
|
|
||
| it.skip("tron", async () => { | ||
| const channelId = "UC5OPOGRq02iK-0T9sKse_kA" | ||
|
|
||
| const response = await getLastVideoDateFromYoutube(channelId) | ||
|
|
||
| expect(response).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/) | ||
| }) | ||
| }) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,44 @@ | ||
| import { getYoutubeChannelLastVideo } from "@/functions/youtube/getYoutubeChannelLastVideo" | ||
| import { getLastSegment } from "@/utils/getLastSegment" | ||
| import { env } from "@/lib/env" | ||
| import { ofetch } from "@/lib/ofetch" | ||
| import { memoize } from "@fxts/core" | ||
| import { z } from "zod" | ||
|
|
||
| export const getLastVideoDateFromYoutube = async (youtubeLink: string) => { | ||
| const channelId = getLastSegment(youtubeLink) | ||
| export const getLastVideoDateFromYoutube = memoize( | ||
| async (youtubeChannelId: string) => { | ||
| const playlistId = `${youtubeChannelId.slice(0, 1)}U${youtubeChannelId.slice(2)}` | ||
|
|
||
| const data = await getYoutubeChannelLastVideo(channelId) | ||
| const response = await ofetch<GetLastVideoDateFromYoutubeResponse>( | ||
| `https://www.googleapis.com/youtube/v3/playlistItems`, | ||
| { | ||
| query: { | ||
| part: "snippet", | ||
| playlistId, | ||
| maxResults: 1, | ||
| key: env.YOUTUBE_DATA_API_KEY, | ||
| }, | ||
| }, | ||
| ) | ||
|
|
||
| const { publishedAt } = data.snippet | ||
| const data = getLastVideoDateFromYoutubeSchema.parse(response) | ||
|
|
||
| return publishedAt | ||
| } | ||
| const { publishedAt } = data.items[0].snippet | ||
|
|
||
| return new Date(publishedAt).toISOString() | ||
| }, | ||
| ) | ||
|
|
||
| const getLastVideoDateFromYoutubeSchema = z.object({ | ||
| items: z | ||
| .array( | ||
| z.object({ | ||
| snippet: z.object({ | ||
| publishedAt: z.string(), | ||
| }), | ||
| }), | ||
| ) | ||
| .min(1), | ||
| }) | ||
|
|
||
| export type GetLastVideoDateFromYoutubeResponse = z.infer< | ||
| typeof getLastVideoDateFromYoutubeSchema | ||
| > |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,11 @@ | ||
| import { getYoutubeChannelInfo } from "@/functions/youtube/getYoutubeChannelInfo" | ||
| import { getLastSegment } from "@/utils/getLastSegment" | ||
|
|
||
| export const getSubscriberCountFromYoutube = async (youtubeLink: string) => { | ||
| const channelId = getLastSegment(youtubeLink) | ||
|
|
||
| export const getSubscriberCountFromYoutube = async ( | ||
| youtubeChannelId: string, | ||
| ) => { | ||
| const { | ||
| statistics: { subscriberCount }, | ||
| } = await getYoutubeChannelInfo(channelId) | ||
| } = await getYoutubeChannelInfo(youtubeChannelId) | ||
|
|
||
| return +subscriberCount | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,9 @@ | ||
| import { getYoutubeChannelInfo } from "@/functions/youtube/getYoutubeChannelInfo" | ||
| import { getLastSegment } from "@/utils/getLastSegment" | ||
|
|
||
| export const getVideoCountFromYoutube = async (youtubeLink: string) => { | ||
| const channelId = getLastSegment(youtubeLink) | ||
|
|
||
| export const getVideoCountFromYoutube = async (youtubeChannelId: string) => { | ||
| const { | ||
| statistics: { videoCount }, | ||
| } = await getYoutubeChannelInfo(channelId) | ||
| } = await getYoutubeChannelInfo(youtubeChannelId) | ||
|
|
||
| return +videoCount | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,11 @@ | ||
| import { getYoutubeChannelId } from "@/functions/youtube/getYoutubeChannelId" | ||
| import { getYoutubeChannelInfo } from "@/functions/youtube/getYoutubeChannelInfo" | ||
| import { getLastSegment } from "@/utils/getLastSegment" | ||
|
|
||
| export const getViewCountFromYoutube = async (youtubeLink: string) => { | ||
| const channelId = getLastSegment(youtubeLink) | ||
| import { youtubeUrlSchema } from "@/validators/youtube" | ||
|
|
||
| export const getViewCountFromYoutube = async (youtubeChannelId: string) => { | ||
| const { | ||
| statistics: { viewCount }, | ||
| } = await getYoutubeChannelInfo(channelId) | ||
| } = await getYoutubeChannelInfo(youtubeChannelId) | ||
|
|
||
| return +viewCount | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| import { getYoutubeChannelId } from "@/functions/youtube/getYoutubeChannelId" | ||
| import { describe, expect, it } from "vitest" | ||
|
|
||
| describe.skip("getYoutubeChannelId", () => { | ||
| it("/channel/<channel-id>", async () => { | ||
| const youtubeUrl = | ||
| "https://www.youtube.com/channel/UC6rYoXJ_3BbPyWx_GQDDRRQ" | ||
|
|
||
| const expected = "UC6rYoXJ_3BbPyWx_GQDDRRQ" | ||
|
|
||
| const response = await getYoutubeChannelId(youtubeUrl) | ||
|
|
||
| expect(response).toBe(expected) | ||
| }) | ||
|
|
||
| it("/@<handle>", async () => { | ||
| const youtubeUrl = "https://youtube.com/@SolanaFndn" | ||
|
|
||
| const expected = "UC9AdQPUe4BdVJ8M9X7wxHUA" | ||
|
|
||
| const response = await getYoutubeChannelId(youtubeUrl) | ||
|
|
||
| expect(response).toBe(expected) | ||
| }) | ||
|
|
||
| it("/c/<custom>", async () => { | ||
| const youtubeUrl = "https://youtube.com/c/Circlecryptofinance" | ||
|
|
||
| const expected = "UCzQMzwoT-Mj_TXggCLUT7Lw" | ||
|
|
||
| const response = await getYoutubeChannelId(youtubeUrl) | ||
|
|
||
| expect(response).toBe(expected) | ||
| }) | ||
|
|
||
| it("/<custom>", async () => { | ||
| const youtubeUrl = "https://youtube.com/MakerDAO" | ||
|
|
||
| const expected = "UC4jqZlzQHUhzqf5rMd5ywTw" | ||
|
|
||
| const response = await getYoutubeChannelId(youtubeUrl) | ||
|
|
||
| expect(response).toBe(expected) | ||
| }) | ||
| }) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| import { env } from "@/lib/env" | ||
| import { ofetch } from "@/lib/ofetch" | ||
| import { getLastSegment } from "@/utils/getLastSegment" | ||
| import { memoize } from "@fxts/core" | ||
| import { z } from "zod" | ||
|
|
||
| export const getYoutubeChannelId = memoize(async (youtubeUrl: string) => { | ||
| const query = getLastSegment(youtubeUrl).replace("@", "") | ||
|
|
||
| const response = await ofetch<GetYoutubeChannelIdResponse>( | ||
| `https://www.googleapis.com/youtube/v3/search`, | ||
| { | ||
| query: { | ||
| type: `channel`, | ||
| part: `snippet`, | ||
| q: query, | ||
| maxResults: 1, | ||
| key: env.YOUTUBE_DATA_API_KEY, | ||
| }, | ||
| }, | ||
| ) | ||
|
|
||
| const data = getYoutubeChannelIdSchema.parse(response) | ||
|
|
||
| return data.items[0].id.channelId | ||
| }) | ||
|
|
||
| const getYoutubeChannelIdSchema = z.object({ | ||
| kind: z.string(), | ||
| etag: z.string(), | ||
| items: z | ||
| .array( | ||
| z.object({ | ||
| kind: z.string(), | ||
| etag: z.string(), | ||
| id: z.object({ | ||
| kind: z.string(), | ||
| channelId: z.string(), | ||
| }), | ||
| }), | ||
| ) | ||
| .min(1), | ||
| }) | ||
|
|
||
| export type GetYoutubeChannelIdResponse = z.infer< | ||
| typeof getYoutubeChannelIdSchema | ||
| > |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,20 @@ | ||
| import { getYoutubeChannelInfo } from "@/functions/youtube/getYoutubeChannelInfo" | ||
| import { describe, expect, it } from "vitest" | ||
|
|
||
| describe("getYoutubeChannelInfo function format tests", () => { | ||
| describe("getYoutubeChannelInfo", () => { | ||
| it("solana", async () => { | ||
| const channelId = "UC9AdQPUe4BdVJ8M9X7wxHUA" | ||
|
|
||
| const response = await getYoutubeChannelInfo(channelId) | ||
|
|
||
| expect(response.id).toBe(channelId) | ||
| }) | ||
|
|
||
| it.skip("near", async () => { | ||
| const channelId = "UCuKdIYVN8iE3fv8alyk1aMw" | ||
|
|
||
| const response = await getYoutubeChannelInfo(channelId) | ||
|
|
||
| expect(response.id).toBe(channelId) | ||
| }) | ||
| }) |
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| import { urlSchema } from "@/validators/urlSchema" | ||
| import { z } from "zod" | ||
|
|
||
| export const youtubeUrlSchema = z.custom<string>((value) => { | ||
| const validUrl = urlSchema.parse(value) | ||
|
|
||
| const { pathname } = new URL(validUrl) | ||
|
|
||
| const channelRegex = /^\/channel\/([A-Za-z0-9_-]+)$/ | ||
| const handleRegex = /^\/@([\w-]+)$/ | ||
| const brandRegex = /^\/c\/([\w-]+)$/ | ||
| const customRegex = /^\/([\w-]+)$/ | ||
|
|
||
| return ( | ||
| channelRegex.test(pathname) || | ||
| handleRegex.test(pathname) || | ||
| brandRegex.test(pathname) || | ||
| customRegex.test(pathname) | ||
| ) | ||
| }) |